/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package parserCrawler;

import dataObjects.webArticle;
import exceptions.notExpectedPageTypeException;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;

/**
 *
 * @author hubert
 */
public class wikiCrawler implements crawler{

    private WebDriver browser;
    private boolean areLinkArticlesCancerArticles;
    private reportingClasses.reportingObject reporter;
    
    public wikiCrawler(){
        this.browser = new org.openqa.selenium.htmlunit.HtmlUnitDriver();
        this.areLinkArticlesCancerArticles = true;
        this.reporter = new reportingClasses.bodyTextWriter("/home/hubert/hadoopStuff/sampleCodeRun/input/isCancer", "/home/hubert/hadoopStuff/sampleCodeRun/input/rest");
    }
    @Override
    public webArticle crawl(String URL, boolean isCancerArticle, int linkDepthToFollow) throws exceptions.notExpectedPageTypeException{
        webArticle serve = new webArticle();
        this.browser.get(URL);
        if(isCancerArticle){
            serve.setIsCancer(isCancerArticle);
        }
        WebElement header = browser.findElement(By.xpath("//*[@id=\"firstHeading\"]/span"));
        WebElement body = browser.findElement(By.xpath("//*[@id=\"mw-content-text\"]"));
        
        //algorithm sourced from: http://learnseleniumautomation.blogspot.com/2012/02/extract-all-links-from-webpage-using.html
        java.util.List<WebElement> rawLinkList = browser.findElement(By.xpath("//*[@id=\"mw-content-text\"]")).findElements(By.tagName("a"));//browser.findElements(By.tagName("a"));
        java.util.ArrayList<dataObjects.hyperLink> linkList = new java.util.ArrayList<dataObjects.hyperLink>();
        for(WebElement link : rawLinkList){
            dataObjects.hyperLink nLink = new dataObjects.hyperLink();
            nLink.setLinkText(link.getText().replace("\n", "").replace("\r", " "));
            nLink.setTargetURL(link.getAttribute("href"));
            if( (nLink.getLinkText().compareTo("edit") != 0) && (nLink.getLinkText().length() > 1) && (!nLink.getLinkText().startsWith("[") && !nLink.getLinkText().endsWith("]"))){
                linkList.add(nLink);
            }
            
        }
        
        if(linkDepthToFollow > 0){
            for(dataObjects.hyperLink link : linkList){
                if((link.getTargetURL().startsWith("http://en.wikipedia.org/wiki"))){
                    java.lang.Thread thrd = new java.lang.Thread();
                    try{thrd.sleep(60000);}catch( java.lang.InterruptedException e){ e.printStackTrace();}
                    
                    wikiCrawler spawnCrawler = new wikiCrawler();
                    spawnCrawler.crawl(link.getTargetURL(), this.isAreLinkArticlesCancerArticles(), (linkDepthToFollow - 1));
                }
                
            }
        }
        
        
        serve.setHyperLinks(linkList);
        java.lang.StringBuilder bodyText = new java.lang.StringBuilder(body.getText().replace("\n", " ").replace("\r", " "));
        bodyText = textCleaning.textCleaner.handleLinkText(bodyText, linkList);
        serve.setArticleText(textCleaning.textCleaner.runAllRemovalFuncs(bodyText).toString());
        serve.setKeyWord(header.getText().replace("/", "slash"));
        serve.setUrl(URL);
        
        this.reporter.report(serve);
        
//        System.out.println("headerText: " + serve.getKeyWord());
//        System.out.println("bodyText\n" + serve.getArticleText());
//        for(dataObjects.hyperLink cLink : serve.getHyperLinks()){
//            System.out.println("text: " + cLink.getLinkText() + ", url: " + cLink.getTargetURL());
//        }
        return serve;
    }
    
    public static void main(String[] args){
        wikiCrawler a = new wikiCrawler();
        reportingClasses.reportingObject rptr = new reportingClasses.bodyTextWriter("/home/hubert/hadoopStuff/sampleCodeRun/input", "/home/hubert/hadoopStuff/sampleCodeRun/input");
        a.setReporter(rptr);
        try{
            a.crawl("http://en.wikipedia.org/wiki/Lung_cancer", true, 0);
        }catch(exceptions.notExpectedPageTypeException e){
            e.printStackTrace();
        }
        
        
        
    }

    /**
     * @return the areLinkArticlesCancerArticles
     */
    public boolean isAreLinkArticlesCancerArticles() {
        return areLinkArticlesCancerArticles;
    }

    /**
     * @param areLinkArticlesCancerArticles the areLinkArticlesCancerArticles to set
     */
    public void setAreLinkArticlesCancerArticles(boolean areLinkArticlesCancerArticles) {
        this.areLinkArticlesCancerArticles = areLinkArticlesCancerArticles;
    }

    /**
     * @return the reporter
     */
    public reportingClasses.reportingObject getReporter() {
        return reporter;
    }

    /**
     * @param reporter the reporter to set
     */
    public void setReporter(reportingClasses.reportingObject reporter) {
        this.reporter = reporter;
    }
    
}
